import org.apache.lucene.analysis.Analyzer;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;


/**
 * Indexer: traverses a file system and indexes .txt files
 *
 * @author Jeremy Rayner <groovy@ross-rayner.com>
 * based on examples in the wonderful 'Lucene in Action' book
 * by Erik Hatcher and Otis Gospodnetic ( http://www.lucenebook.com )
 *
 * requires a lucene-1.x.x.jar from http://lucene.apache.org
 */


def indexPath = new File("C:\\Users\\laurie\\Java\\indexes") // Create Lucene index in this directory
def dataDir = new File("C:\\Users\\laurie\\Documents\\datasets\\test") // Index files in this directory

def start = new Date().time
//def numIndexed = index(indexDir, dataDir)
def end = new Date().time

//println "Indexing $numIndexed files took ${end - start} milliseconds"

def index(indexDir, dataDir) {
	if (!dataDir.exists() || !dataDir.directory) {
		throw new IOException("$dataDir does not exist or is not a directory")
	}
	
	def Directory dir = FSDirectory.open(new File(indexPath));
	def     Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_41);
	def      IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_41, analyzer)
	
	def create = true
	if (create) {
	       // Create a new index in the directory, removing any
	       // previously indexed documents:
	      iwc.setOpenMode(OpenMode.CREATE);
	     } else {
	       // Add new documents to an existing index:
	       iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
	     }
	
		// iwc.setRAMBufferSizeMB(256.0);
		 
		 IndexWriter writer = new IndexWriter(dir, iwc);
		 
//	def writer = new IndexWriter(indexDir, new StandardAnalyzer(), true) // Create Lucene index
//	writer.useCompoundFile = false

	dataDir.eachFileRecurse {
		if (it.name =~ /.txt$/) { // Index .txt files only
			indexFile(writer,it)
		}
	}
	def numIndexed = writer.docCount()
	writer.optimize()
	writer.close() // Close index
	return numIndexed
}

void indexFile(writer, f) {
	if (f.hidden || !f.exists() || !f.canRead() || f.directory) { return }
	
	Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_41);

	println "Indexing $f.canonicalPath"
	def doc = new Document()

	// Construct a Field that is tokenized and indexed, but is not stored in the index verbatim.
	doc.add(Field.Text("contents", new FileReader(f)))

	// Construct a Field that is not tokenized, but is indexed and stored.
	doc.add(Field.Keyword("filename",f.canonicalPath))

	writer.addDocument(doc) // Add document to Lucene index
}